﻿/*======================================================================
== Copyright : BlueCurve (c)
== Licence   : Gnu/GPL v2.x
== Author    : Teddy Albina
== Email     : bluecurveteam@gmail.com
== Web site  : http://www.codeplex.com/BlueCurve
========================================================================*/
using System;
using System.AddIn;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Cache;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Xml.Linq;
using BlueCurve.AddinView;
using BlueCurve.DownloadAttributes;
using BlueCurve.EnumAction;
using BlueCurve.WebCrawlerRequest;
using RobotRules;

namespace Addins.Http
{
    /// <summary>
    /// Addin permettant d'effetuer des requêtes Http
    /// </summary>
    [AddIn("AddinsHttp",
    Description = "This addin can do HTTP request",
    Version = "1.0.0.0",
    Publisher = "BlueCurve (c) Project Team")]
    public class AddinsHttp : AddinProtocolViewAddinsSide
    {
        #region 'Configuration'

        /// <summary>
        /// Parseur de fichier robots.txt
        /// </summary>
        private RobotsFileParser RobotRules = new RobotsFileParser();
        /// <summary>
        /// Retourne/définit les type de fichiers acceptés
        /// </summary>
        public override string Accept { get; set; }
        /// <summary>
        /// Retourne/définit l'adresse d'un eventuel proxy
        /// </summary>
        public override Uri Proxy { get; set; }
        /// <summary>
        /// Type mime accepté par defaut
        /// </summary>
        private const string DefaultAccept = @"application/xml+xhtml text/html application/x-httpd-php text/plain application/zip multipart/x-zip multipart/x-gzip application/pdf";

        #endregion



        #region 'Function'
         
        /// <summary>
        /// Effectue des requetes HTTP
        /// </summary>
        /// <param name="downloadata">Données concernant le document à télécharger</param>
        /// <param name="certificates">Certificats X509</param>
        /// <param name="certificates2">Certificats X509V2</param>
        /// <returns>BlueCurve.WebCrawlerRequest.RequestState</returns>
        public override RequestState Download(BlueCurve.DownloadAttributes.Attributes downloadata, X509Certificate[] certificates, X509Certificate[] certificates2)
        {
            BlueCurve.WebCrawlerRequest.RequestState rs = new BlueCurve.WebCrawlerRequest.RequestState()
            {
                Options = downloadata,
                TypeOffRequest = BlueCurve.WebCrawlerRequest.RequestType.http,
                Uri = new Uri(downloadata.Url),
                Action = downloadata.Action
            };
            try
            {
                // Vérification de l'accès au document
                if (!CheckRobotsRules(new Uri(downloadata.Url)))
                {
                    switch (rs.Action)
                    {
                        case IndexAction.UPDATE: 
                            rs.Action = IndexAction.DELETE;
                            return rs;
                        case IndexAction.INDEX:
                        case IndexAction.DELETE: return null;
                    }
                }

                // Configuration de la requete HTTP
                HttpWebRequest webrequest = (HttpWebRequest)WebRequest.Create(new Uri(downloadata.Url));
                webrequest.AllowAutoRedirect = true;
                webrequest.CookieContainer = new CookieContainer() { Capacity = 200, MaxCookieSize = 12048, PerDomainCapacity = 20 };
                webrequest.KeepAlive = true;
                webrequest.Timeout = 10 * 1000;
                webrequest.MaximumAutomaticRedirections = 1;
                webrequest.UserAgent = @"Mozilla/5.0 (compatible; Bluebot/1.0; +http://www.codeplex.com/BlueCurve)";
                webrequest.CachePolicy = new RequestCachePolicy(System.Net.Cache.RequestCacheLevel.Revalidate);
                webrequest.AutomaticDecompression = DecompressionMethods.None | DecompressionMethods.Deflate | DecompressionMethods.GZip;
                //webrequest.ContinueDelegate = new HttpContinueDelegate(ContinueDelegate);

                if (!string.IsNullOrEmpty(Accept))
                    webrequest.Accept = Accept;
                else
                    webrequest.Accept = DefaultAccept;

                this.MakeAccept(downloadata.Options);

                if (Proxy != null)
                    this.MakeWebProxy(downloadata.Options, ref webrequest);

                if (certificates != null && certificates.Length > 0)
                    webrequest.ClientCertificates = new X509CertificateCollection(certificates);

                if (certificates2 != null && certificates2.Length > 0)
                    webrequest.ClientCertificates = new X509CertificateCollection(certificates2);

                // Besoin d'une authentification ?
                NetworkCredential credential = BlueCurve.Common.Crawlers.Authentification.MakeUriCredential(downloadata.Options);
                if (credential != null)
                    webrequest.Credentials = credential;
                
                #region 'Head'

                // Objet de la requête head
                HttpWebResponse webheadresponse = DoHeadRequest(ref downloadata, ref certificates, ref certificates2);

                if (webheadresponse != null)
                {
                    if (!string.IsNullOrEmpty(Accept))
                    {
                        if (!Accept.Contains(webheadresponse.Headers[HttpResponseHeader.ContentType].ToLower()))
                        {
                            return null; // Type mime non authorisé
                        }
                    }

                    webrequest.ProtocolVersion = webheadresponse.ProtocolVersion;
                }
                else
                {
                    webrequest.ProtocolVersion = HttpVersion.Version11;
                }

                #endregion


                // Récuperation de la reponse
                HttpWebResponse webresponse = (HttpWebResponse)webrequest.GetResponse();
                // Sauvegarde des propriétés de la réponse dans l'objet RequestState
                rs.CharacterSet    = webresponse.CharacterSet;
                rs.ContentLength   = webresponse.ContentLength;
                rs.ContentType     = webresponse.ContentType;
                rs.Headers         = webresponse.Headers;
                rs.IsFromCache     = webresponse.IsFromCache;
                rs.LastModified    = webresponse.LastModified;
                rs.ProtocolVersion = webresponse.ProtocolVersion;
                rs.ResponseUri     = webresponse.ResponseUri;
                rs.StatusCode      = webresponse.StatusCode;

                // Lecture des données
                Stream responseStream = webresponse.GetResponseStream();
                using (BinaryReader writer = new BinaryReader(responseStream, Encoding.UTF8))
                {
                    rs.SetBuffer(writer.ReadBytes((int)webresponse.ContentLength));
                    writer.Close();
                }
                if (responseStream != null)
                    responseStream.Close();

                switch (webresponse.StatusCode)
                {
                    case HttpStatusCode.BadGateway:
                    case HttpStatusCode.BadRequest:
                    case HttpStatusCode.Conflict:
                    case HttpStatusCode.Forbidden:
                    case HttpStatusCode.ExpectationFailed:
                    case HttpStatusCode.GatewayTimeout:
                    case HttpStatusCode.Gone:
                    case HttpStatusCode.HttpVersionNotSupported:
                    case HttpStatusCode.InternalServerError:
                    case HttpStatusCode.MethodNotAllowed:
                    case HttpStatusCode.NoContent:
                    case HttpStatusCode.NonAuthoritativeInformation:
                    case HttpStatusCode.NotAcceptable:
                    case HttpStatusCode.NotFound:
                    case HttpStatusCode.NotImplemented:
                    case HttpStatusCode.NotModified:
                    case HttpStatusCode.PaymentRequired:
                    case HttpStatusCode.PreconditionFailed:
                    case HttpStatusCode.RequestedRangeNotSatisfiable:
                    case HttpStatusCode.RequestTimeout:
                    case HttpStatusCode.RequestUriTooLong:
                    case HttpStatusCode.ServiceUnavailable:
                    case HttpStatusCode.Unauthorized:
                    case HttpStatusCode.UnsupportedMediaType:
                    case HttpStatusCode.SwitchingProtocols:
                    case HttpStatusCode.ResetContent:
                        if (rs.Action == IndexAction.INDEX || rs.Action == IndexAction.UPDATE)
                            rs.Action = IndexAction.DELETE;
                        break;
                    case HttpStatusCode.SeeOther:
                    case HttpStatusCode.Ambiguous:
                        this.Download(new Uri(rs.Headers[HttpResponseHeader.Location]), ref rs, ref downloadata, ref certificates, ref certificates2);
                        break;
                    default:
                        break;
                }

                return rs;
            }
            catch
            {
                return null;
            }
        }
        
        
        /// <summary>
        /// Effectue des requetes HTTP
        /// </summary>
        /// <param name="url">Nouvelle url</param>
        /// <param name="rs">Objet RequestState</param>
        /// <param name="downloadata">Données concernant le document à télécharger</param>
        /// <param name="certificates">Certificats X509</param>
        /// <param name="certificates2">Certificats X509V2</param>
        /// <returns>BlueCurve.WebCrawlerRequest.RequestState</returns>
        private void Download(Uri url, ref BlueCurve.WebCrawlerRequest.RequestState rs, ref BlueCurve.DownloadAttributes.Attributes downloadata, ref System.Security.Cryptography.X509Certificates.X509Certificate[] certificates, ref System.Security.Cryptography.X509Certificates.X509Certificate[] certificates2)
        {
            try
            {
                // Vérification de l'accès au document
                if (!CheckRobotsRules(url))
                {
                    rs.Action = IndexAction.DELETE;
                    return;
                }

                // Configuration de la requete HTTP
                HttpWebRequest webrequest = (HttpWebRequest)WebRequest.Create(new Uri(downloadata.Url));
                webrequest.AllowAutoRedirect = true;
                webrequest.CookieContainer = new CookieContainer() { Capacity = 200, MaxCookieSize = 12048, PerDomainCapacity = 20 };
                webrequest.KeepAlive = true;
                webrequest.Timeout = 10 * 1000; // ten seconds
                webrequest.MaximumAutomaticRedirections = 1;
                webrequest.UserAgent = @"Mozilla/5.0 (compatible; Bluebot/1.0; +http://www.codeplex.com/BlueCurve)";
                webrequest.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.Revalidate);
                webrequest.AutomaticDecompression = DecompressionMethods.None | DecompressionMethods.Deflate | DecompressionMethods.GZip;

                if (!string.IsNullOrEmpty(Accept))
                    webrequest.Accept = Accept;
                else
                    webrequest.Accept = DefaultAccept;

                this.MakeAccept(downloadata.Options);

                if (Proxy != null)
                    this.MakeWebProxy(downloadata.Options, ref webrequest);

                if (certificates != null && certificates.Length > 0)
                    webrequest.ClientCertificates = new X509CertificateCollection(certificates);

                if (certificates2 != null && certificates2.Length > 0)
                    webrequest.ClientCertificates = new X509CertificateCollection(certificates2);

                // Besoin d'une authentification ?
                NetworkCredential credential = BlueCurve.Common.Crawlers.Authentification.MakeUriCredential(downloadata.Options);
                if (credential != null)
                    webrequest.Credentials = credential;

                // Objet de la requête head
                HttpWebResponse webheadresponse = DoHeadRequest(ref downloadata, ref certificates, ref certificates2);

                if (webheadresponse != null)
                {
                    if (!string.IsNullOrEmpty(Accept))
                    {
                        if (!Accept.Contains(webheadresponse.Headers[HttpResponseHeader.ContentType].ToLower()))
                        {
                            rs = null; // Type mime non authorisé
                        }
                    }

                    webrequest.ProtocolVersion = webheadresponse.ProtocolVersion;
                }
                else
                {
                    webrequest.ProtocolVersion = HttpVersion.Version11;
                }

                // On lance la requête
                HttpWebResponse webresponse = (HttpWebResponse)webrequest.GetResponse();
                // Sauvegarde des propriétés de la réponse dans l'objet RequestState
                rs.CharacterSet = webresponse.CharacterSet;
                rs.ContentLength = webresponse.ContentLength;
                rs.ContentType = webresponse.ContentType;
                rs.Headers = webresponse.Headers;
                rs.IsFromCache = webresponse.IsFromCache;
                rs.LastModified = webresponse.LastModified;
                rs.ProtocolVersion = webresponse.ProtocolVersion;
                rs.ResponseUri = webresponse.ResponseUri;
                rs.StatusCode = webresponse.StatusCode;

                // Lecture des données
                Stream responseStream = webresponse.GetResponseStream();
                using (BinaryReader writer = new BinaryReader(responseStream, Encoding.UTF8))
                {
                    rs.SetBuffer(writer.ReadBytes((int)webresponse.ContentLength));
                    writer.Close();
                }
                if (responseStream != null)
                    responseStream.Close();


                switch (webresponse.StatusCode)
                {
                    case HttpStatusCode.BadGateway:
                    case HttpStatusCode.BadRequest:
                    case HttpStatusCode.Conflict:
                    case HttpStatusCode.Forbidden:
                    case HttpStatusCode.ExpectationFailed:
                    case HttpStatusCode.GatewayTimeout:
                    case HttpStatusCode.Gone:
                    case HttpStatusCode.HttpVersionNotSupported:
                    case HttpStatusCode.InternalServerError:
                    case HttpStatusCode.MethodNotAllowed:
                    case HttpStatusCode.NoContent:
                    case HttpStatusCode.NonAuthoritativeInformation:
                    case HttpStatusCode.NotAcceptable:
                    case HttpStatusCode.NotFound:
                    case HttpStatusCode.NotImplemented:
                    case HttpStatusCode.NotModified:
                    case HttpStatusCode.PaymentRequired:
                    case HttpStatusCode.PreconditionFailed:
                    case HttpStatusCode.RequestedRangeNotSatisfiable:
                    case HttpStatusCode.RequestTimeout:
                    case HttpStatusCode.RequestUriTooLong:
                    case HttpStatusCode.ServiceUnavailable:
                    case HttpStatusCode.Unauthorized:
                    case HttpStatusCode.UnsupportedMediaType:
                    case HttpStatusCode.SwitchingProtocols:
                    case HttpStatusCode.ResetContent:
                        if (rs.Action == IndexAction.INDEX || rs.Action == IndexAction.UPDATE)
                            rs.Action = IndexAction.DELETE;
                        break;
                    case HttpStatusCode.SeeOther:
                    case HttpStatusCode.Ambiguous:
                        rs = null;
                        break;
                    default:
                        break;
                }
            }
            catch (Exception e)
            {
                Console.WriteLine(e);
            }
        }
        

        /// <summary>
        /// Fonction attaché au délégué de la requête
        /// </summary>
        /// <param name="StatusCode">Code du statut</param>
        /// <param name="httpHeaders">En-tete HTTP</param>
        private void ContinueDelegate(int StatusCode, WebHeaderCollection httpHeaders)
        {
            /// How it's work ???
        }


        /// <summary>
        /// Effectue une requête Http de type HEAD
        /// </summary>
        /// <param name="downloadata">Données pour l'accès au document</param>
        /// <returns>HttpWebResponse</returns>
        private HttpWebResponse DoHeadRequest(ref BlueCurve.DownloadAttributes.Attributes downloadata, ref System.Security.Cryptography.X509Certificates.X509Certificate[] certificates, ref System.Security.Cryptography.X509Certificates.X509Certificate[] certificates2)
        {
            try
            {
                HttpWebRequest webrequest = (HttpWebRequest)WebRequest.Create(new Uri(downloadata.Url));
                webrequest.AllowAutoRedirect = true;
                webrequest.CookieContainer = new CookieContainer() { Capacity = 200, MaxCookieSize = 12048, PerDomainCapacity = 20 };
                webrequest.KeepAlive = true;
                webrequest.MaximumAutomaticRedirections = 1;
                webrequest.UserAgent = @"Mozilla/5.0 (compatible; Bluebot/1.0; +http://www.codeplex.com/BlueCurve)";
                webrequest.Timeout = 10 * 1000;
                webrequest.Method = "HEAD";
                webrequest.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.Revalidate);

                if (!string.IsNullOrEmpty(Accept))
                    webrequest.Accept = Accept;
                else
                    webrequest.Accept = DefaultAccept;

                this.MakeAccept(downloadata.Options);

                if (Proxy != null)
                    this.MakeWebProxy(downloadata.Options, ref webrequest);

                if (certificates != null && certificates.Length > 0)
                    webrequest.ClientCertificates = new X509CertificateCollection(certificates);

                if (certificates2 != null && certificates2.Length > 0)
                    webrequest.ClientCertificates = new X509CertificateCollection(certificates2);

                // Besoin d'une authentification ?
                NetworkCredential credential = BlueCurve.Common.Crawlers.Authentification.MakeUriCredential(downloadata.Options);
                if (credential != null)
                    webrequest.Credentials = credential;

                return ((HttpWebResponse)webrequest.GetResponse());
            }
            catch
            {
                return null;
            }
        }

       


        /// <summary>
        /// Fabrique un objet de type webproxy
        /// </summary>
        /// <param name="data">Données à utiliser pour la conception du proxy</param>
        /// <param name="webrequest">Réference vers l'objet HttpWebRequest en cours</param>
        private void MakeWebProxy(string data, ref HttpWebRequest webrequest)
        {
            WebProxy proxy = new WebProxy(Proxy);
            try
            {
                NetworkCredential credential = new NetworkCredential();

                XDocument doc = XDocument.Load(new StringReader(data));

                var cred = from x in doc.Descendants("proxy")
                           select new
                           {
                               Bypassforlocal = x.Element("bypassforlocal").Value,
                               Default = x.Element("default").Value,
                               Computer = x.Element("computer").Value,
                               Password = x.Element("password").Value,
                               Id = x.Element("id").Value
                           };

                foreach (var x in cred)
                {
                    if (!string.IsNullOrEmpty(x.Default) && x.Default.Equals("true", StringComparison.InvariantCultureIgnoreCase))
                    {
                        proxy.UseDefaultCredentials = true;
                    }
                    else
                    {
                        if (!string.IsNullOrEmpty(x.Id))
                        {
                            credential.UserName = x.Id;
                            if (!string.IsNullOrEmpty(x.Password))
                                credential.Password = x.Password;
                            if (!string.IsNullOrEmpty(x.Computer))
                                credential.Domain = x.Computer;

                            proxy.Credentials = credential;
                        }
                    }
                    if (!string.IsNullOrEmpty(x.Bypassforlocal))
                        proxy.BypassProxyOnLocal = bool.Parse(x.Bypassforlocal);
                }

                // Set the proxy
                webrequest.Proxy = proxy;
            }
            catch
            {
                webrequest.Proxy = proxy;
            }
        }

        
        /// <summary>
        /// Renvoie une liste des types mime acceptés
        /// </summary>
        /// <param name="data">Fichier XML</param>
        private void MakeAccept(string data)
        {
            try
            {
                if (!string.IsNullOrEmpty(data))
                {
                    XDocument doc = XDocument.Load(new StringReader(data));

                    var v = from p in doc.Descendants("accept")
                            select p.Element("mime").Value.ToLower(System.Globalization.CultureInfo.InvariantCulture);

                    foreach (string x in v)
                    {
                        string[] mimes = x.Split('|');

                        for (int i = 0; i < mimes.Length; i++)
                        {
                            if (!Accept.Contains(mimes[i]))
                                string.Concat(Accept, " ", mimes[i]);
                        }
                    }
                }
            }
            catch (Exception e)
            {
                Console.WriteLine(e);
            }
        }


        /// <summary>
        /// Vérifie si le lien peut être téléchargé
        /// </summary>
        /// <param name="uri">Uri du document</param>
        /// <returns>bool</returns>
        private bool CheckRobotsRules(Uri uri)
        {
            this.RobotRules.Parse(uri);
            return this.RobotRules.IsAllowed("bluebot", uri);
        }

        #endregion
    }
}
